# ----- Always run this at the beginning of class so we can get multi-command output ----- #

# Access options from the iPython core
from IPython.core.interactiveshell import InteractiveShell

# Change the value of ast_node_interactivity
InteractiveShell.ast_node_interactivity = "all"

# ----- Additional packages we want to import for class ----- #

# Import the pandas package
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np

# seaborn should already be installed on Jupyter Hub
# !pip install seaborn

# Read in embryo_long_merged.csv
embryo_merged = pd.read_csv('data/embryo_long_merged.csv') 

# Take a peek at the data
embryo_merged.head()

# How big is this dataset?
embryo_merged.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 11149 entries, 0 to 11148
Data columns (total 31 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   worm.number        11149 non-null  int64  
 1   date               11149 non-null  int64  
 2   wormStrain         11149 non-null  object 
 3   pathogenStrain     11149 non-null  object 
 4   pathogenDose       11149 non-null  object 
 5   doseLevel          11149 non-null  object 
 6   timepoint          11149 non-null  object 
 7   merontsPresent     11149 non-null  bool   
 8   sporesPresent      11149 non-null  bool   
 9   numEmbryos         11149 non-null  int64  
 10  experiment         11149 non-null  object 
 11  experimenter       11149 non-null  object 
 12  description        11149 non-null  object 
 13  Infection Date     11149 non-null  int64  
 14  Plate Number       11149 non-null  int64  
 15  Total Worms        11149 non-null  int64  
 16  Spore Lot          11149 non-null  object 
 17  Lot concentration  11149 non-null  int64  
 18  Total ul spore     11149 non-null  float64
 19  Infection Round    11149 non-null  int64  
 20  40X OP50 (mL)      11149 non-null  float64
 21  Plate Size         11149 non-null  int64  
 22  Spores/cm2         11149 non-null  float64
 23  Temp               11149 non-null  int64  
 24  infection.type     11149 non-null  object 
 25  Staining Date      11149 non-null  int64  
 26  Stain type         11149 non-null  object 
 27  Slide date         11149 non-null  int64  
 28  Slide number       11149 non-null  int64  
 29  Slide Box          11149 non-null  int64  
 30  Imaging Date       11149 non-null  int64  
dtypes: bool(2), float64(3), int64(15), object(11)
memory usage: 2.5+ MB

# Subset the data from our dataset for the first 19 columns
embryo_merged_subset = embryo_merged.iloc[:, :19]

# What does the subset look like?
embryo_merged_subset.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 11149 entries, 0 to 11148
Data columns (total 19 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   worm.number        11149 non-null  int64  
 1   date               11149 non-null  int64  
 2   wormStrain         11149 non-null  object 
 3   pathogenStrain     11149 non-null  object 
 4   pathogenDose       11149 non-null  object 
 5   doseLevel          11149 non-null  object 
 6   timepoint          11149 non-null  object 
 7   merontsPresent     11149 non-null  bool   
 8   sporesPresent      11149 non-null  bool   
 9   numEmbryos         11149 non-null  int64  
 10  experiment         11149 non-null  object 
 11  experimenter       11149 non-null  object 
 12  description        11149 non-null  object 
 13  Infection Date     11149 non-null  int64  
 14  Plate Number       11149 non-null  int64  
 15  Total Worms        11149 non-null  int64  
 16  Spore Lot          11149 non-null  object 
 17  Lot concentration  11149 non-null  int64  
 18  Total ul spore     11149 non-null  float64
dtypes: bool(2), float64(1), int64(7), object(9)
memory usage: 1.5+ MB

# Get a description of the numeric data
embryo_merged_subset.describe()

# Use the describe method on the OTU column from our merged_subset
embryo_merged_subset.loc[:,['wormStrain', 'pathogenStrain']].describe()

# Recall that we can broadcast a conditional query to multiple values in a DataFrame
embryo_merged_subset.loc[:, 'pathogenDose'] > 0

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[6], line 2
      1 # Recall that we can broadcast a conditional query to multiple values in a DataFrame
----> 2 embryo_merged_subset.loc[:, 'pathogenDose'] > 0

File ~\anaconda3\envs\CSBJupyter\Lib\site-packages\pandas\core\ops\common.py:76, in _unpack_zerodim_and_defer.<locals>.new_method(self, other)
     72             return NotImplemented
     74 other = item_from_zerodim(other)
---> 76 return method(self, other)

File ~\anaconda3\envs\CSBJupyter\Lib\site-packages\pandas\core\arraylike.py:56, in OpsMixin.__gt__(self, other)
     54 @unpack_zerodim_and_defer("__gt__")
     55 def __gt__(self, other):
---> 56     return self._cmp_method(other, operator.gt)

File ~\anaconda3\envs\CSBJupyter\Lib\site-packages\pandas\core\series.py:5803, in Series._cmp_method(self, other, op)
   5800 lvalues = self._values
   5801 rvalues = extract_array(other, extract_numpy=True, extract_range=True)
-> 5803 res_values = ops.comparison_op(lvalues, rvalues, op)
   5805 return self._construct_result(res_values, name=res_name)

File ~\anaconda3\envs\CSBJupyter\Lib\site-packages\pandas\core\ops\array_ops.py:346, in comparison_op(left, right, op)
    343     return invalid_comparison(lvalues, rvalues, op)
    345 elif lvalues.dtype == object or isinstance(rvalues, str):
--> 346     res_values = comp_method_OBJECT_ARRAY(op, lvalues, rvalues)
    348 else:
    349     res_values = _na_arithmetic_op(lvalues, rvalues, op, is_cmp=True)

File ~\anaconda3\envs\CSBJupyter\Lib\site-packages\pandas\core\ops\array_ops.py:131, in comp_method_OBJECT_ARRAY(op, x, y)
    129     result = libops.vec_compare(x.ravel(), y.ravel(), op)
    130 else:
--> 131     result = libops.scalar_compare(x.ravel(), y, op)
    132 return result.reshape(x.shape)

File ops.pyx:107, in pandas._libs.ops.scalar_compare()

TypeError: '>' not supported between instances of 'str' and 'int'

# Use the insert method
embryo_merged_subset.insert(loc = 4,                   # Location to insert at
                            column = 'pathogenDose',   # The "new" column name to insert
                            
                            # To calculate the value, we'll pop from the subset (which removes the chosen column!)
                            value = (embryo_merged_subset.pop('pathogenDose')
                            
                                     # Break up the float from the "M" which will make 2 columns
                                     .str.split(pat = "M", expand = True)
                            
                                     # Convert the first column to a float, and then provide only that to insert                                    
                                     .astype({0:'float64'})[0])    # The [0] index slices our DataFrame returns just the first column as a Series
                           )

# Check on the updated subset data
embryo_merged_subset.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 11149 entries, 0 to 11148
Data columns (total 19 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   worm.number        11149 non-null  int64  
 1   date               11149 non-null  int64  
 2   wormStrain         11149 non-null  object 
 3   pathogenStrain     11149 non-null  object 
 4   pathogenDose       11149 non-null  float64
 5   doseLevel          11149 non-null  object 
 6   timepoint          11149 non-null  object 
 7   merontsPresent     11149 non-null  bool   
 8   sporesPresent      11149 non-null  bool   
 9   numEmbryos         11149 non-null  int64  
 10  experiment         11149 non-null  object 
 11  experimenter       11149 non-null  object 
 12  description        11149 non-null  object 
 13  Infection Date     11149 non-null  int64  
 14  Plate Number       11149 non-null  int64  
 15  Total Worms        11149 non-null  int64  
 16  Spore Lot          11149 non-null  object 
 17  Lot concentration  11149 non-null  int64  
 18  Total ul spore     11149 non-null  float64
dtypes: bool(2), float64(2), int64(7), object(8)
memory usage: 1.5+ MB

# Now we can use a conditional comparison on the pathogenDose column
embryo_merged_subset.loc[:, 'pathogenDose'] > 0

0        False
1        False
2        False
3        False
4        False
         ...  
11144     True
11145     True
11146     True
11147     True
11148     True
Name: pathogenDose, Length: 11149, dtype: bool

# Supply the result of the conditional query as a filtering criteria
print("Filtered pathogenStrain summary")
embryo_merged_subset.loc[embryo_merged_subset.loc[:, 'pathogenDose'] > 0, # remember we're filtering by rows!
                         # We only need the pathogenStrain column to summarize
                         'pathogenStrain'   # No list notation means we'll get a Series back 
                        ].describe()

# Note the \n adds a blank line to our output
print("\nUnfiltered pathogenStrain summary")

# Compare that to the unfiltered subset
embryo_merged_subset.loc[:, 'pathogenStrain'].describe()

Filtered pathogenStrain summary

count      7730
unique       10
top       LUAm1
freq       2838
Name: pathogenStrain, dtype: object

Unfiltered pathogenStrain summary

count     11149
unique       10
top       LUAm1
freq       5030
Name: pathogenStrain, dtype: object

# Filter for experiments with actual pathogen exposure
pathogen_Summary = embryo_merged_subset.loc[embryo_merged_subset.loc[:, 'pathogenDose'] > 0,
                         # We only need the pathogenStrain column to summarize
                         'pathogenStrain'].describe()

# Of the LUAm1 infections, which worm strain reigns supreme?
embryo_merged_subset.loc[(embryo_merged_subset["pathogenStrain"] == pathogen_Summary.top), # Filter rows by the "top" pathogenStrain 
                         'wormStrain'].describe()                                          # Keep the wormStrain column to summarize it

count     5030
unique      15
top         N2
freq       864
Name: wormStrain, dtype: object

# Select the wormStrain, pathogenStrain, and numEmbryos columns
embryo_merged_subset.filter(items = ['wormStrain', 'pathogenStrain', 'numEmbryos'])

# Filter to see which pathogen strain is most often measured.
# We'll encapsulate the call in () so we can separate across lines

# Filter the dataset for observations with actual infection
(embryo_merged_subset.query('pathogenDose > 0')
                     # Select just pathogenStrain as a column
                     .filter(['pathogenStrain'])
                     # Get a summary
                     .describe()
)

# To complete our query, we use the @ to identify our intermediate variable
(embryo_merged_subset.query('pathogenStrain == @pathogen_Summary.top')
                     # Filter on the column we want to use
                     .filter(["wormStrain"])
                     # Summarize
                     .describe()
)

# How many individual dates/replicates are there?
len(embryo_merged_subset.loc[:,'date'].unique())

11

# Group our subset data by the 'date' column
embryo_merged_subset.groupby(by = ['date'])

# What is returned to us?

<pandas.core.groupby.generic.DataFrameGroupBy object at 0x000001AE860C82D0>

# Group our subset data by the 'date' column and view 1 row from each group
embryo_merged_subset.groupby(by = ['date']).head(1)

# Group our subset data by the 'date' and 'wormStrain' columns and grab the first row from each
# How big is the result?
embryo_merged_subset.groupby(by = ['date', 'wormStrain', 'pathogenStrain']).head(1).shape

(99, 19)

# Convert the "wormStrain" column from our grouped data to a frequency table
(embryo_merged_subset.groupby(by = ['date', 'wormStrain', 'pathogenStrain'])
                     # head(1) gets us a representative row from each group
                     .head(1)
                     # Create the frequency table
                     .value_counts(subset = ['wormStrain'])
)

wormStrain
N2            29
JU1400        28
MY1            7
AWR145         6
AWR144         6
VC40171        3
VC20019        3
ED3052A        3
ED3052B        3
MY2            2
JU360          2
JU642          1
JU397          1
JU300          1
MY6            1
ED3042         1
CB4856         1
AB1            1
Name: count, dtype: int64

# Identify the mean embryo value of uninfected strains

# Subset by uninfected animals
(embryo_merged_subset.query('pathogenDose == 0')
 
                     # Group by wormStrain and take the numEmbryos columns
                     .groupby(by = ['wormStrain'])['numEmbryos']
                     
                     # Calculate the mean
                     .mean()
)

wormStrain
AB1        10.903226
AWR144     19.228000
AWR145     20.656000
CB4856     20.750000
ED3042     13.147541
ED3052A    10.526667
ED3052B    12.047297
JU1400     11.139501
JU300      15.622951
JU360      16.788618
JU397      11.457143
JU642      14.733333
MY1        11.725581
MY2        21.886179
MY6        18.360656
N2         18.912500
VC20019    15.143678
VC40171     4.586667
Name: numEmbryos, dtype: float64

# Identify the mean embryo value of uninfected strains AND sort them!

# Subset by uninfected animals
(embryo_merged_subset.query('pathogenDose == 0')
 
                     # Group by wormStrain and take the numEmbryos colums
                     .groupby(by = ['wormStrain'])['numEmbryos']
                     
                     # Calculate the mean
                     .mean()
                     
                     # Sort the data by descending order
                     .sort_values(ascending = False)
)

wormStrain
MY2        21.886179
CB4856     20.750000
AWR145     20.656000
AWR144     19.228000
N2         18.912500
MY6        18.360656
JU360      16.788618
JU300      15.622951
VC20019    15.143678
JU642      14.733333
ED3042     13.147541
ED3052B    12.047297
MY1        11.725581
JU397      11.457143
JU1400     11.139501
AB1        10.903226
ED3052A    10.526667
VC40171     4.586667
Name: numEmbryos, dtype: float64

# Comprehension code answer 1.3.1
# Identify the mean embryo value of uninfected strains AND sort them!

# Subset by uninfected animals
(embryo_merged_subset.query('pathogenDose == 0')
 
                     # Group by wormStrain and take the numEmbryos columns
                     .groupby(by = ['wormStrain'])['numEmbryos']
                     
                     # Calculate the mean
                     .mean()
                     
                     # How do we calculate the median?
                     ...
)

# Determine the standard deviation of the mean embryo counts across each strain
# IN the uninfected state - ie a baseline embryo count.

# Query for only uninfected data
(embryo_merged_subset.query('pathogenDose == 0')
 
 # Group by infection experiment 
 .groupby(by = ['date', 'wormStrain', 'pathogenStrain'])
 
 # Isolate numEmbryos and generate a mean for each group
 # Indexing like this returns a SeriesGroupBy object
 ['numEmbryos']
 .mean()
 
 # Group the series of means again by wormStrain
 .groupby(['wormStrain'])
 
 # Summarize each group of numEmbryo means
 .describe()
 
 # Sort the data by descending order
 .sort_values(by = 'mean', ascending = False)
)

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Isolate pathogen strain information in each group
 ['pathogenStrain']
 
 # Determine the unique values in each
 .unique()
)

wormStrain
AB1                                                  [LUAm1]
AWR144                                        [LUAm1, ERTm5]
AWR145                                        [LUAm1, ERTm5]
CB4856                                               [ERTm5]
ED3042                                               [LUAm1]
ED3052A                                              [LUAm1]
ED3052B                                              [LUAm1]
JU1400     [LUAm1, ERTm5, AWRm78, LUAm3, MAM1, LUAm1-HK, ...
JU300                                                [ERTm5]
JU360                                         [LUAm1, ERTm2]
JU397                                                [LUAm1]
JU642                                                [LUAm1]
MY1                                           [LUAm1, ERTm5]
MY2                                           [ERTm5, ERTm2]
MY6                                                  [LUAm1]
N2         [LUAm1, ERTm5, ERTm2, AWRm78, LUAm3, MAM1, LUA...
VC20019                                [LUAm1, ERTm5, ERTm2]
VC40171                                              [LUAm1]
Name: pathogenStrain, dtype: object

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Isolate pathogen strain information in each group
 ['pathogenStrain']
 
 # Determine the unique values in each
 .unique()
 
 # Extract the size of each array
 .size
)

18

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Isolate pathogen strain information in each group
 ['pathogenStrain']
 
 # Determine the unique values in each
 .unique()
 
 # Extract the size of each array
 .apply(lambda x: x.size)
)

wormStrain
AB1         1
AWR144      2
AWR145      2
CB4856      1
ED3042      1
ED3052A     1
ED3052B     1
JU1400      9
JU300       1
JU360       2
JU397       1
JU642       1
MY1         2
MY2         2
MY6         1
N2         10
VC20019     3
VC40171     1
Name: pathogenStrain, dtype: int64

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Determine the length of unique values in each
 .nunique()
)

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Isolate pathogen strain information in each group
 ['pathogenStrain']
 
 # Determine the total number of values in each group, the number of unique, and what they are
 .agg([lambda x: x.size, "count", "nunique", "unique"])
)

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Determine the total number of values in each group, the number of unique, and what they are
 .agg({'worm.number':['count'], 
       'numEmbryos':['mean', 'std']})
)

# Query for only infected data
(embryo_merged_subset.query('wormStrain in ["N2", "JU1400"] & pathogenDose == 0')
 
 # Group our data by worm strain
 .groupby('wormStrain')
 
 # Determine the number of observations in each group, and summary stats
 .agg({'worm.number':['count'], 
       'numEmbryos':['mean', 'std']})
)

# Comprehension code answer 1.5.6

# Query for only infected data
(embryo_merged_subset.query('pathogenDose > 0')
 # Group our data by worm strain
 .groupby(by = ['pathogenStrain', 'pathogenDose'])

 # Determine the total number of unique worm strain values in each group, the number of unique, and what they are
 .agg(...)
 
 # Sort the values in nunique to simplify your search
 .sort_values(...)
)

# We'll reset the code cells to only show the last code call. This will de-clutter the plotting process for us
InteractiveShell.ast_node_interactivity = "last"

# Determine the mean number of embryos across all non-infected observations per strain
wormStrain_mean_embryos = (embryo_merged_subset.query('pathogenDose == 0')
                           
 # Group data by worm strains 
 .groupby('wormStrain')
                           
 # Get the mean value in each group
 .agg({'numEmbryos':'mean'})

 # Sort the data
 .sort_values(by = 'numEmbryos', ascending = False)
)

# Check on the results
wormStrain_mean_embryos

# Build our barplot by giving the index as the x-label, and values as the height
# We'll need to supply the data directly to each parameter
plt.bar(x = wormStrain_mean_embryos.index, 
        height = wormStrain_mean_embryos['numEmbryos'])

# Show our plot
plt.show()

## 2.1.1 Fix the size of the plot
plt.figure(figsize = (12,5))

# Build our barplot
plt.bar(x=wormStrain_mean_embryos.index, 
        height=wormStrain_mean_embryos['numEmbryos'])

# Show our plot
plt.show()

# Fix the size of the plot
plt.figure(figsize = (12, 5))

# Build our barplot
plt.bar(x=wormStrain_mean_embryos.index, 
        height=wormStrain_mean_embryos['numEmbryos'])

## 2.1.2 Rotate the x-axis text
plt.xticks(rotation = 90)

# Show our plot
plt.show()

# Fix the size of the plot
plt.figure(figsize = (12, 5))

# Build our barplot
plt.bar(x=wormStrain_mean_embryos.index, 
        height=wormStrain_mean_embryos['numEmbryos'])

# Rotate the x-axis text
plt.xticks(rotation = 90)

## 2.1.3 Add titles
plt.title("Baseline mean embryos per worm strain", fontsize = "x-large") 
# You can play with font style
plt.xlabel("worm strain", fontstyle = "italic")
# And even the font colour!
plt.ylabel("mean embryo count", color = "r")

# Show our plot
plt.show()

# Fix the size of the plot
plt.figure(figsize = (12, 5))

# Build our barplot
plt.bar(x=wormStrain_mean_embryos.index, 
        height=wormStrain_mean_embryos['numEmbryos'],
        ## 2.1.4 you can also change some barplot aspects
        color="orchid",
        edgecolor="black")

# Rotate the x-axis text
plt.xticks(rotation = 90)

# Add titles
plt.title("Baseline mean embryos per worm strain", fontsize = "x-large") 
plt.xlabel("worm strain", fontstyle = "italic")
plt.ylabel("mean embryo count", color = "r")

# Show our plot
plt.show()

?plt.axis

# Comprehension answer code 2.0.0
# Fix the size of the plot
plt.figure(figsize = (12, 5))

# Build our barplot
plt.bar(x=wormStrain_mean_embryos.index, 
        height=wormStrain_mean_embryos['numEmbryos'],
        color="orchid",
        edgecolor="black")

# Rotate the x-axis text
plt.xticks(rotation = 90)

# Add titles
plt.title("Baseline mean embryos per worm strain", fontsize = "x-large") 
plt.xlabel("worm strain", fontstyle = "italic")
plt.ylabel("mean embryo count", color = "r")

# Alter the axis limits
plt.axis(...)

# Show our plot
plt.show()

# We'll reset the code cells to only show FINAL code output.
InteractiveShell.ast_node_interactivity = "last"

# Read the pitlatrine data in from file
infectionSig_data = pd.read_csv("data/infection_signal.tsv", sep = '\t')

# Look at the first 5 rows of data
infectionSig_data.head(5)

# How big is this dataframe?
infectionSig_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 456 entries, 0 to 455
Data columns (total 14 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   exp.name          456 non-null    object 
 1   strain            456 non-null    object 
 2   spore.strain      456 non-null    object 
 3   spore.species     456 non-null    object 
 4   dose              456 non-null    object 
 5   spores            456 non-null    float64
 6   fixing.date       456 non-null    object 
 7   slide             456 non-null    int64  
 8   file              456 non-null    object 
 9   worm.number       456 non-null    int64  
 10  area              456 non-null    float64
 11  percent.infected  456 non-null    float64
 12  area.infected     456 non-null    float64
 13  timepoint         456 non-null    object 
dtypes: float64(4), int64(2), object(8)
memory usage: 50.0+ KB

# What are the unique values in each column?
infectionSig_data.apply(pd.unique, axis = 0)

exp.name            [N2-LUAm1-1.8, JU1400-LUAm1-1.8, AWR144-LUAm1-...
strain                                   [N2, JU1400, AWR144, AWR145]
spore.strain                                                  [LUAm1]
spore.species                                         [N.ferruginous]
dose                                                      [pulse-72H]
spores                                                          [1.8]
fixing.date                                        [rep1, rep2, rep3]
slide                                                             [1]
file                [N2.LUAm1.rep1, JU1400.LUAm1.rep1, AWR144.LUAm...
worm.number         [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14...
area                [49838.02, 50425.04, 45532.67, 46458.55, 49214...
percent.infected    [18.53, 0.0, 31.16, 3.88, 23.54, 1.65, 8.6, 18...
area.infected       [9234.985106, 0.0, 14187.97997, 1802.59174, 93...
timepoint                                                     [72hpi]
dtype: object

# Import your seaborn package
import seaborn as sns

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   # Set the data
                      height = 6, aspect = 1,   # Set the size of the figure
                      kind = "scatter"          # Set the figure type
                     )

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,  
                      x = "area", y = "area.infected",     # Set the axis variables
                      height = 6, aspect = 1,   
                      kind = "scatter"          
                     )

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "strain",          # Set the point-colour by strain
                      alpha = 0.6,              # Set the transparency of the points
                      height = 6, aspect = 1,  
                      kind = "scatter"          
                     )

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "strain",          # Set the point-colour by strain
                      alpha = 0.6,              # Set the transparency of the points
                      height = 6, aspect = 1,  
                      kind = "scatter"          
                     )

## 3.2.2 Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      alpha = 0.6,               # Set the transparency of the points
                      height = 6, aspect = 0.6,  # Set the size of the figure
                      kind = "scatter",
                      col = "strain"             ## 3.2.3 Split the columns of the grid by country
                     )

# Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "percent.infected", palette = "Reds",   ## 3.2.4 Set the point-colour by percent infected
                      edgecolor = "black",       ## 3.2.4 Set the point border colour so we can see them all
                      alpha = 0.6,               
                      height = 6, aspect = 0.6,  
                      kind = "scatter",
                      col = "strain"             # Split the columns of the grid by country
                     )

# Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "percent.infected", palette = "Reds",   # Set the point-colour by percent infected
                      edgecolor = "black",       # Set the point border colour so we can see them all
                      alpha = 0.6,               
                      height = 6, aspect = 0.6,  
                      kind = "scatter",
                      col = "strain"             # Split the columns of the grid by country
                     )

## 3.2.5 Set the axis titles (Aesthetics)
snsPlot.set_axis_labels(x_var = "Total area", y_var = "Area infected")

# Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "percent.infected", palette = "Reds",   # Set the point-colour by percent infected
                      edgecolor = "black",       # Set the point border colour so we can see them all
                      style = "fixing.date",     # 3.2.6 Change the style of our points based on variable values
                      alpha = 0.6,               
                      height = 6, aspect = 0.5,  
                      kind = "scatter",
                      col = "strain"             # Split the columns of the grid by country
                     )

# Set the axis titles (Aesthetics)
snsPlot.set_axis_labels(x_var = "Total area", y_var = "Area infected")

# Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# We'll build a scatterplot
snsPlot = sns.relplot(data = infectionSig_data,   
                      x = "area", y = "area.infected",     
                      hue = "percent.infected",  # Set the point-colour by percent infected
                      palette = sns.color_palette(palette = "viridis_r", 
                                                  as_cmap = True),  ## 3.2.7 Use a better colour palette like viridis
                      edgecolor = "black",       # Set the point border colour so we can see them all
                      style = "fixing.date",     # 3.2.6 Change the style of our points based on variable values
                      alpha = 0.7,               
                      height = 6, aspect = 0.5,  
                      kind = "scatter",
                      col = "strain"             # Split the columns of the grid by country
                     )

# Set the axis titles (Aesthetics)
snsPlot.set_axis_labels(x_var = "Total area", y_var = "Area infected")

# Change your y-axis to a log scale
snsPlot.set(yscale="log")

# Show the plot (avoid extra object information)
plt.show()

# Filter the data by N2 animals with a pathogenDose of 0
N2_mock_data = embryo_merged_subset.query('wormStrain == "N2" & pathogenDose == 0')

# Filter code for using loc instead:
# N2_mock_data = embryo_merged_subset.loc[(embryo_merged_subset['wormStrain'] == "N2") &
#                                         (embryo_merged_subset['pathogenDose'] == 0), 
#                                        :]

# Check on the data created
N2_mock_data.info()

<class 'pandas.core.frame.DataFrame'>
Index: 720 entries, 503 to 10748
Data columns (total 19 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   worm.number        720 non-null    int64  
 1   date               720 non-null    int64  
 2   wormStrain         720 non-null    object 
 3   pathogenStrain     720 non-null    object 
 4   pathogenDose       720 non-null    float64
 5   doseLevel          720 non-null    object 
 6   timepoint          720 non-null    object 
 7   merontsPresent     720 non-null    bool   
 8   sporesPresent      720 non-null    bool   
 9   numEmbryos         720 non-null    int64  
 10  experiment         720 non-null    object 
 11  experimenter       720 non-null    object 
 12  description        720 non-null    object 
 13  Infection Date     720 non-null    int64  
 14  Plate Number       720 non-null    int64  
 15  Total Worms        720 non-null    int64  
 16  Spore Lot          720 non-null    object 
 17  Lot concentration  720 non-null    int64  
 18  Total ul spore     720 non-null    float64
dtypes: bool(2), float64(2), int64(7), object(8)
memory usage: 102.7+ KB

# We'll build a KDE plot
snsPlot = sns.displot(data = N2_mock_data,          
                      x = "numEmbryos",
                      hue = "date", 
                      height = 6, aspect = 2,        
                      kind = "kde",
                      fill = True,             # The fill parameter is passed on to kdeplot()
                     )
# Show the plot (avoid extra object information)
plt.show()

# Convert our date variable with the astype() method
N2_mock_data = N2_mock_data.astype({'date':'category'})

# rebuild a KDE plot
snsPlot = sns.displot(data = N2_mock_data,          
                      x = "numEmbryos",
                      hue = "date", 
                      height = 6, aspect = 2,        
                      kind = "kde",
                      fill = True,             # The fill parameter is passed on to kdeplot()
                     )
# Show the plot (avoid extra object information)
plt.show()

# Take the unique values and flip the order
date_list = N2_mock_data.date.unique()[::-1].tolist()

# View the reversed date list
date_list

[200916,
 200905,
 200918,
 200915,
 200904,
 200825,
 200821,
 200721,
 200714,
 200707,
 190426]

# We'll need to pull out and replace the date column
# In this case we will not "pop" the column out but work on it as part of the DataFrame
N2_mock_data['date'] = (N2_mock_data['date']
                        # access the categorical property
                        .cat
                        # Reorder the categorical data
                        .reorder_categories(new_categories = date_list)
                       )

# Double-check the date column results
N2_mock_data['date']

503      190426
504      190426
505      190426
506      190426
507      190426
          ...  
10744    200916
10745    200916
10746    200916
10747    200916
10748    200916
Name: date, Length: 720, dtype: category
Categories (11, int64): [200916, 200905, 200918, 200915, ..., 200721, 200714, 200707, 190426]

# Rebuild a KDE plot with the new category order
snsPlot = sns.displot(data = N2_mock_data,          
                      x = "numEmbryos",
                      hue = "date",            # Now the date variable is a category!
                      height = 6, aspect = 2,        
                      kind = "kde",
                      fill = True,             # The fill parameter is passed on to kdeplot()
                     )
# Show the plot (avoid extra object information)
plt.show()

# We'll build a KDE plot
snsPlot = sns.displot(data = N2_mock_data,          
                      x = "numEmbryos",
                      hue = "date",            # Now the date variable is a category!
                      height = 6, aspect = 2,        
                      kind = "kde",
                      fill = True,             # The fill parameter is passed on to kdeplot()
                     )

## 3.3.4 Add a rugplot - this is plotted on top of the current axes object
sns.rugplot(data = N2_mock_data,
            x = "numEmbryos", 
            height = -0.02,         # Draw the rug plot BELOW the x-axis
            clip_on = False,        # Ensure that it is rendered
            alpha = 0.5
           )

# Show the plot (avoid extra object information)
plt.show()

# filter by doseLevel
mean_embryo_data = (embryo_merged_subset.query('doseLevel in ["Mock", "Medium"]')

                    # Group by infection experiment 
                    .groupby(by = ['date', 'wormStrain', 'pathogenStrain', 'doseLevel'])

                    # Create the frequency table on numEmbryos
                    # ['numEmbryos']
                    .agg({'numEmbryos':'mean'})

                    # Recall, when we reset the index, it converts the indices back into columns
                    .reset_index()

                    # Set our wormStrain and doseLevel variables as categories!
                    .astype({'wormStrain':'category', 'doseLevel':'category'})
)

# Check out our summarized data
mean_embryo_data

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos",                       
                      kind = "box", # Make a boxplot
                      height = 6, aspect = 2 # Set the width of our plot to 4 and width at 12 (6x2)
                     )
                    
# Show the plot (avoid extra object information)
plt.show()

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos", 
                      kind = "box",                        # Make a boxplot
                      height = 6, aspect = 2,
                      hue = "wormStrain",                  # Colour our wormStrains
                     )

## 3.4.2 Alter the xtick attributes
snsPlot.set_xticklabels(rotation = 90)

# Show the plot (avoid extra object information)
plt.show()

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos", 
                      kind = "box",                        # Make a boxplot
                      height = 6, aspect = 2,
                      hue = "doseLevel",         ## 3.4.4 Set the hue by doseLevel
                      legend_out = False,        ## 3.4.4 Move the legend inside the plot
                      width = 0.6                ## 3.4.4 Put some more distance between categories by decreasing width
                     )

# Alter the xtick attributes
snsPlot.set_xticklabels(rotation = 90)                    

# Show the plot (avoid extra object information)
plt.show()

# Pull out our doseLevel data and replace it with a newly categorized version
mean_embryo_data['doseLevel'] = (mean_embryo_data['doseLevel']
                                 .cat
                                 .reorder_categories(new_categories = ["Mock", "Medium"])
                                )

# Double check it worked
mean_embryo_data['doseLevel'].cat.categories

Index(['Mock', 'Medium'], dtype='object')

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos", 
                      kind = "box",                        # Make a boxplot
                      height = 6, aspect = 2,
                      hue = "doseLevel",         # Set the hue by doseLevel
                      legend_out = False,        # Move the legend inside the plot
                      width = 0.6                # Put some more distance between categories by decreasing width
                     )

# Alter the xtick attributes
snsPlot.set_xticklabels(rotation = 90)                    

# Show the plot (avoid extra object information)
plt.show()

sns.set(font_scale = 4)

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "doseLevel", y = "numEmbryos", 
                      kind = "box",              # Make a boxplot
                      height = 10, aspect = 1,
                      width = 0.6,               # Put some more distance between categories by decreasing width
                      col = "wormStrain",        ## 3.4.5 Set our facets based on wormStrain 
                      col_wrap = 5               ## 3.4.5 Only use 5 facets per row
                     )

## 3.4.5 Alter the xtick attributes
snsPlot.set_xticklabels(labels = ["Mock", "Medium"])                    
               
## 3.4.5 Change our y-axis label
snsPlot.set_axis_labels("dose level", "embryos per animal")

## 3.4.5 set the title
snsPlot.set_titles("{col_name}")

# Show the plot (avoid extra object information)
plt.show()

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "doseLevel", y = "numEmbryos", 
                      kind = "box",                        # Make a boxplot
                      height = 10, aspect = 1,
                      width = 0.6,                # Put some more distance between categories by decreasing width
                      hue = "doseLevel",          ## 3.4.6 Re-colour our boxes by doseLevel again!
                      col = "wormStrain",
                      col_wrap = 5,
                      fliersize = 0               ## 3.4.6 Hide our outliers by making them size 0
                     )

# Alter the xtick attributes
snsPlot.set_xticklabels(labels = ["Mock", "Medium"])                    
               
# set the title
snsPlot.set_titles("{col_name}")

## 3.4.6 Overlay a swarmplot on our catplot
snsPlot.map_dataframe(sns.swarmplot, data = mean_embryo_data,
                      x = "doseLevel", 
                      y = "numEmbryos", 
                      hue = "doseLevel",        # split the points by dose level just like nested boxplots
                      palette="dark:black",     # Recolour all of the points to black
                      size=15                   # Set the size of our points so they can be seen
)

## 3.4.6 Change our y-axis label. It appears to work best AFTER the map_dataframe() call
snsPlot.set_axis_labels("dose level", "embryos per animal")

# Show the plot (avoid extra object information)
plt.show()

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "doseLevel", y = "numEmbryos", 
                      kind = "violin",            ## 3.5.0 Make a violin plot
                      height = 10, aspect = 1,
                      width = 0.6,                # Put some more distance between categories by decreasing width
                      hue = "doseLevel",          # Re-colour our violins by doseLevel again!
                      col = "wormStrain",
                      col_wrap = 5
                     )

# Alter the xtick attributes
snsPlot.set_xticklabels(labels = ["Mock", "Medium"])                    
               
# set the title
snsPlot.set_titles("{col_name}")

# Overlay a swarmplot on our catplot
snsPlot.map_dataframe(sns.swarmplot, data = mean_embryo_data,
                      x = "doseLevel", 
                      y = "numEmbryos", 
                      hue = "doseLevel",        # split the points by dose level just like nested boxplots
                      palette="dark:black",     # Recolour all of the points to black
                      size=15                   # Set the size of our points so they can be seen
)

# Change our y-axis label. It appears to work best AFTER the map_dataframe() call
snsPlot.set_axis_labels("dose level", "embryos per animal")

# Show the plot (avoid extra object information)
plt.show()

sns.set(font_scale = 1)

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos", 
                      kind = "violin",                        # Make a boxplot
                      height = 6, aspect = 2, 
                      hue = "doseLevel",                      
                      split = True,            ## 3.5.1 This will create hybrid violin plots
                      inner = "quart",         ## 3.5.1 Add quartile markers to each half of the violin
                      density_norm="width",    ## 3.5.1 Make all the widths the same
                      palette = {"Mock":"green", "Medium":"yellow"}  ## 3.5.1 Update the palette of our violins
                     )

## 3.5.1 Overlay a stripplot on our catplot
snsPlot.map_dataframe(sns.stripplot, data = mean_embryo_data,
                      x = "wormStrain", 
                      y = "numEmbryos", 
                      hue = "doseLevel",        # split the points by dose level just like nested boxplots
                      palette="dark:black",     # Recolour all of the points to black
                      size=5,                   # Set the size of our points so they can be seen
                      dodge = True              # Reduce the overlap on any points
)

# Change our y-axis label. It appears to work best AFTER the map_dataframe() call
snsPlot.set_axis_labels("dose level", "embryos per animal")

# Alter the xtick attributes
snsPlot.set_xticklabels(rotation = 90)  

# Show the plot (avoid extra object information)
plt.show()

# Use catplot to make our boxplot
snsPlot = sns.catplot(data = mean_embryo_data,
                      x = "wormStrain", y = "numEmbryos", 
                      kind = "violin",                        # Make a boxplot
                      height = 6, aspect = 2,
                      hue = "doseLevel",
                      split = True,            # This will create hybrid violin plots
                      inner = "quart",         # Add quartile markers to each half of the violin
                      density_norm="width",    # Make all the widths the same
                      palette = {"Mock":"green", "Medium": "yellow"}  # Update the palette of our violins
                     )

# Overlay a swarmplot on our catplot
snsPlot.map_dataframe(sns.stripplot, data = mean_embryo_data,
                      x = "wormStrain", 
                      y = "numEmbryos", 
                      hue = "doseLevel",        # split the points by dose level just like nested boxplots
                      palette="dark:black",     # Recolour all of the points to black
                      size=5,                   # Set the size of our points so they can be seen
                      dodge = True
)

# Change our y-axis label. It appears to work best AFTER the map_dataframe() call
snsPlot.set_axis_labels("dose level", "embryos per animal")

# Alter the xtick attributes
snsPlot.set_xticklabels(rotation = 90)  

# Save the plot
plt.savefig("data/mean_embryos_byStrain.png", 
            format = "png", 
            dpi = 300
           )

# Initialize a figure
fig = plt.figure()

# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid(shape = (2, 2), loc = (0,), colspan = 1, rowspan = 2)
ax2 = plt.subplot2grid(shape = (2, 2), loc = (...), colspan=1)
ax3 = plt.subplot2grid(shape = (2, 2), loc = (...), colspan=1)

plt.show()

# Initialize a figure
fig = plt.figure(figsize=(15, 15))
# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=2)
ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=2)
ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=2)

# ---------- Plot 1 ----------#
## 5.1.1 Add the first plot - scatterplot
# We'll build a scatterplot
...(data = infectionSig_data,   
                x = "area", 
                y = "area.infected",     
                hue = "strain",          # Set the point-colour by strain
                alpha = 0.6,             # Set the transparency of the points
                ax = ...                 # Set this plot to ax1 within the fig object
               )

# Directly set the labels on the x-axis and y-axis in ax1
...set_xlabel("total area")
...set_ylabel("area infected")

# Show the plot
plt.show()

# Initialize a figure
fig = plt.figure(figsize=(15, 15))
# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=2)
ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=2)
ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=2)

# ---------- Plot 1 ----------#
# Add the first plot - scatterplot
# We'll build a scatterplot
sns.scatterplot(data = infectionSig_data,   
                x = "area", 
                y = "area.infected",     
                hue = "strain",          # Set the point-colour by strain
                alpha = 0.6,             # Set the transparency of the points
                ax = ax1                 # Set this plot to ax1 within the fig object
               )

# Directly set the labels on the x-axis and y-axis of ax1
ax1.set_xlabel("total area")
ax1.set_ylabel("area infected")

# ---------- Plot 2 ----------#
# Add the second plot - violin plot
## 5.1.2 Use violinplot to make our split violin
...(data = mean_embryo_data,
               x = "wormStrain", y = "numEmbryos", 
               hue = "doseLevel",
               split = True,              # This will create hybrid violin plots
               inner = "quartile",        # Add quartile markers to each half of the violin
               palette = {"Mock":"green", "Medium": "yellow"},    # Update the palette of our violins
               density_norm="width",      # Make all the widths the same
               alpha = 0.6,
               ax = ...                   # Set this plot to ax2 within the fig object
              )

# Overlay a swarmplot on our catplot
sns.stripplot(data = mean_embryo_data,
              x = "wormStrain", 
              y = "numEmbryos", 
              hue = "doseLevel",        # split the points by dose level just like nested boxplots
              palette="dark:black",     # Recolour all of the points to black
              size=5,                   # Set the size of our points so they can be seen
              dodge = True,
              ax = ...
             )

# Directly set the labels on the x-axis and y-axis of ax2
ax2.set_xlabel("worm strain")
ax2.set_ylabel("embryos per animal")

# 5.1.2 Alter the xtick attributes
ax2...(axis = 'x', rotation = 90)

# Show the plot
plt.show()

# Initialize a figure
fig = plt.figure(figsize=(15, 15))
# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=2)
ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=2)
ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=2)

# ---------- Plot 1 ----------#
# Add the first plot - scatterplot
# We'll build a scatterplot
sns.scatterplot(data = infectionSig_data,   
                x = "area", 
                y = "area.infected",     
                hue = "strain",          # Set the point-colour by strain
                alpha = 0.6,             # Set the transparency of the points
                ax = ax1                 # Set this plot to ax1 within the fig object
               )

# Directly set the labels on the x-axis and y-axis of ax1
ax1.set_xlabel("total area")
ax1.set_ylabel("area infected")

# ---------- Plot 2 ----------#
# Add the second plot - violin plot
# Use violinplot to make our split violin
sns.violinplot(data = mean_embryo_data,
               x = "wormStrain", y = "numEmbryos", 
               hue = "doseLevel",
               split = True,              # This will create hybrid violin plots
               inner = "quartile",        # Add quartile markers to each half of the violin
               palette = {"Mock":"green", "Medium": "yellow"},    # Update the palette of our violins
               density_norm="width",      # Make all the widths the same
               alpha = 0.6,
               ax = ax2                   # Set this plot to ax2 within the fig object
              )

sns.move_legend(ax2, ...)       ## 5.1.3 Move the legend to the lower right portion of the plot

# Overlay a swarmplot on our catplot
sns.stripplot(data = mean_embryo_data,
              x = "wormStrain", 
              y = "numEmbryos", 
              hue = "doseLevel",        # split the points by dose level just like nested boxplots
              palette="dark:black",     # Recolour all of the points to black
              size=5,                   # Set the size of our points so they can be seen
              dodge = True,
              legend = False,           ## 5.1.3 Set the legend to false to remove it from the plot
              ax = ax2
             )

# Directly set the labels on the x-axis and y-axis of ax2
ax2.set_xlabel("worm strain")
ax2.set_ylabel("embryos per animal")

# Alter the xtick attributes
ax2.tick_params(axis = 'x', rotation = 90)

# Show the plot
plt.show()

# Initialize a figure
fig = plt.figure(figsize=(15, 15))
# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=2)
ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=2)
ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=2)

# ---------- Plot 1 ----------#
# Add the first plot - scatterplot
# We'll build a scatterplot
sns.scatterplot(data = infectionSig_data,   
                x = "area", 
                y = "area.infected",     
                hue = "strain",          # Set the point-colour by strain
                alpha = 0.6,             # Set the transparency of the points
                ax = ax1                 # Set this plot to ax1 within the fig object
               )

# Directly set the labels on the x-axis and y-axis of ax1
ax1.set_xlabel("total area")
ax1.set_ylabel("area infected")

# ---------- Plot 2 ----------#
# Add the second plot - violin plot
# Use violinplot to make our split violin
sns.violinplot(data = mean_embryo_data,
               x = "wormStrain", y = "numEmbryos", 
               hue = "doseLevel",
               split = True,              # This will create hybrid violin plots
               inner = "quartile",        # Add quartile markers to each half of the violin
               palette = {"Mock":"green", "Medium": "yellow"},    # Update the palette of our violins
               density_norm="width",      # Make all the widths the same
               alpha = 0.6,
               ax = ax2                   # Set this plot to ax2 within the fig object
              )

sns.move_legend(ax2, "lower right")       ## 5.1.3 Move the legend to the lower right portion of the plot

# Overlay a swarmplot on our catplot
sns.stripplot(data = mean_embryo_data,
              x = "wormStrain", 
              y = "numEmbryos", 
              hue = "doseLevel",        # split the points by dose level just like nested boxplots
              palette="dark:black",     # Recolour all of the points to black
              size=5,                   # Set the size of our points so they can be seen
              dodge = True,
              legend = False,           ## 5.1.3 Set the legend to false to remove it from the plot
              ax = ax2
             )

# Directly set the labels on the x-axis and y-axis of ax2
ax2.set_xlabel("worm strain")
ax2.set_ylabel("embryos per animal")

# Alter the xtick attributes
ax2.tick_params(axis = 'x', rotation = 90)

# ---------- Plot 3 ----------#
# 5.1.4 rebuild a KDE plot
...(data = N2_mock_data,                      
            x = "numEmbryos",
            hue = "date", 
            fill = True,             # The fill parameter is passed on to kdeplot()
            ax = ax3                 # Set this plot to ax2 within the fig object
           )

# Directly set the x-axis label
ax3.set_xlabel("mean embryos per animal")

# Show the plot
plt.show()

# Initialize a figure
fig = plt.figure(figsize=(15, 15))
# Generate 3 subplots onto our current figure
ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=2)
ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=2)
ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=2)

# ---------- Plot 1 ----------#
# Add the first plot - scatterplot
# We'll build a scatterplot
sns.scatterplot(data = infectionSig_data,   
                x = "area", 
                y = "area.infected",     
                hue = "strain",          # Set the point-colour by strain
                alpha = 0.6,             # Set the transparency of the points
                ax = ax1                 # Set this plot to ax1 within the fig object
               )

# Directly set the labels on the x-axis and y-axis of ax1
ax1.set_xlabel("total area")
ax1.set_ylabel("area infected")

# ---------- Plot 2 ----------#
# Add the second plot - violin plot
# Use violinplot to make our split violin
sns.violinplot(data = mean_embryo_data,
               x = "wormStrain", y = "numEmbryos", 
               hue = "doseLevel",
               split = True,              # This will create hybrid violin plots
               inner = "quartile",        # Add quartile markers to each half of the violin
               palette = {"Mock":"green", "Medium": "yellow"},    # Update the palette of our violins
               density_norm="width",      # Make all the widths the same
               alpha = 0.6,
               ax = ax2                   # Set this plot to ax2 within the fig object
              )

sns.move_legend(ax2, "lower right")       ## 5.1.3 Move the legend to the lower left portion of the plot

# Overlay a swarmplot on our catplot
sns.stripplot(data = mean_embryo_data,
              x = "wormStrain", 
              y = "numEmbryos", 
              hue = "doseLevel",        # split the points by dose level just like nested boxplots
              palette="dark:black",     # Recolour all of the points to black
              size=5,                   # Set the size of our points so they can be seen
              dodge = True,
              legend = False,           ## 5.1.3 Set the legend to false to remove it from the plot
              ax = ax2
             )

# Directly set the labels on the x-axis and y-axis of ax2
ax2.set_xlabel("worm strain")
ax2.set_ylabel("embryos per animal")

# Alter the xtick attributes
ax2.tick_params(axis = 'x', rotation = 90)

# ---------- Plot 3 ----------#
# rebuild a KDE plot
sns.kdeplot(data = N2_mock_data,                      
            x = "numEmbryos",
            hue = "date", 
            fill = True,             # The fill parameter is passed on to kdeplot()
            ax = ax3                 # Set this plot to ax2 within the fig object
           )

# Directly set the x-axis label
ax3.set_xlabel("mean embryos per animal")

## 5.1.5 Re-adjust the axes to remove overlap due to axis text
...

# Show the plot
plt.show()

	worm.number	date	numEmbryos	Infection Date	Plate Number	Total Worms	Lot concentration	Total ul spore
count	11149.000000	11149.00000	11149.000000	11149.000000	11149.000000	11149.000000	11149.00000	11149.000000
mean	27.499148	197408.14378	9.148444	197405.090143	18.785721	1309.444793	273701.61001	25.916820
std	16.460217	4864.97377	7.509851	4864.935158	15.939817	1678.485600	124153.58178	33.928541
min	1.000000	190426.00000	0.000000	190423.000000	1.000000	500.000000	63625.00000	0.000000
25%	14.000000	190426.00000	2.000000	190423.000000	7.000000	1000.000000	176000.00000	0.000000
50%	27.000000	200714.00000	9.000000	200711.000000	14.000000	1000.000000	176000.00000	8.196721
75%	40.000000	200825.00000	15.000000	200822.000000	25.000000	1000.000000	427000.00000	56.818182
max	115.000000	200918.00000	48.000000	200915.000000	63.000000	10000.000000	427000.00000	113.636364

	count	mean	std	min	25%	50%	75%	max
wormStrain
MY2	2.0	21.892460	0.364216	21.634921	21.763690	21.892460	22.021230	22.150000
CB4856	1.0	20.750000	NaN	20.750000	20.750000	20.750000	20.750000	20.750000
AWR145	5.0	20.656000	2.160435	17.180000	20.520000	20.860000	21.760000	22.960000
AWR144	5.0	19.228000	1.990407	16.940000	17.400000	20.060000	20.100000	21.640000
N2	13.0	18.795973	2.816125	15.240000	16.300000	19.358209	20.880000	24.541667
MY6	1.0	18.360656	NaN	18.360656	18.360656	18.360656	18.360656	18.360656
JU360	2.0	16.799841	1.952303	15.419355	16.109598	16.799841	17.490085	18.180328
JU300	1.0	15.622951	NaN	15.622951	15.622951	15.622951	15.622951	15.622951
VC20019	3.0	15.181159	0.616995	14.468750	15.000000	15.531250	15.537364	15.543478
JU642	1.0	14.733333	NaN	14.733333	14.733333	14.733333	14.733333	14.733333
ED3042	1.0	13.147541	NaN	13.147541	13.147541	13.147541	13.147541	13.147541
ED3052B	3.0	12.020833	2.987960	10.062500	10.301250	10.540000	13.000000	15.460000
MY1	4.0	11.611538	1.578067	9.520000	10.960000	11.840000	12.491538	13.246154
JU397	1.0	11.457143	NaN	11.457143	11.457143	11.457143	11.457143	11.457143
JU1400	12.0	11.217214	3.025072	5.724638	9.470000	11.720000	13.585000	14.660000
AB1	1.0	10.903226	NaN	10.903226	10.903226	10.903226	10.903226	10.903226
ED3052A	3.0	10.526667	0.761665	9.700000	10.190000	10.680000	10.940000	11.200000
VC40171	3.0	4.586667	1.571284	2.980000	3.820000	4.660000	5.390000	6.120000

	worm.number	numEmbryos
	count	mean	std
wormStrain
AB1	120	6.433333	4.154037
AWR144	300	7.743333	4.980644
AWR145	300	2.196667	3.098925
CB4856	125	14.656000	6.054995
ED3042	107	1.598131	2.794401
ED3052A	183	2.568306	3.124607
ED3052B	200	2.280000	2.695912
JU1400	2109	4.055477	4.829862
JU300	132	6.651515	3.021513
JU360	251	3.920319	5.707681
JU397	113	9.380531	3.480349
JU642	121	7.446281	4.151205
MY1	613	5.389886	6.395696
MY2	250	9.036000	8.560833
MY6	112	9.294643	5.625935
N2	2221	10.373706	6.531484
VC20019	323	5.170279	5.222870
VC40171	150	0.026667	0.230164

Plot type	Command	What to use it for
Bar plot / histogram	`bar()`	Population data summaries. Helpful for contrasting between groups
Scatter plot	`scatter()`	Multiple independent measurements across different variables
Line plot	`plot()`	Multiple measurements that represent the same sample(s)
Histogram	`hist()`	Generate a distribution by binning your data
Stem or lollipop	`stem()`	A twist on the bar plot that may be more compact and visually pleasing
Boxplots	`boxplot()`	Create a visual summary of your datapoints based on their distribution
Violin plots	`violinplot()`	Create a visual kernel density (distribution) estimate of datapoints

	worm.number	date	wormStrain	pathogenStrain	pathogenDose	doseLevel	timepoint	merontsPresent	sporesPresent	numEmbryos	...	Plate Size	Temp	infection.type	Staining Date	Stain type	Slide date	Slide number	Slide Box	Imaging Date
0	1	190426	AB1	LUAm1	0M	Mock	72hpi	False	False	10	...	6	21	continuous	190430	DY96	190501	7	2	190502
1	2	190426	AB1	LUAm1	0M	Mock	72hpi	False	False	9	...	6	21	continuous	190430	DY96	190501	7	2	190502
2	3	190426	AB1	LUAm1	0M	Mock	72hpi	False	False	16	...	6	21	continuous	190430	DY96	190501	7	2	190502
3	4	190426	AB1	LUAm1	0M	Mock	72hpi	False	False	13	...	6	21	continuous	190430	DY96	190501	7	2	190502
4	5	190426	AB1	LUAm1	0M	Mock	72hpi	False	False	8	...	6	21	continuous	190430	DY96	190501	7	2	190502

	<lambda_0>	count	nunique	unique
wormStrain
AB1	120	120	1	[LUAm1]
AWR144	300	300	2	[LUAm1, ERTm5]
AWR145	300	300	2	[LUAm1, ERTm5]
CB4856	125	125	1	[ERTm5]
ED3042	107	107	1	[LUAm1]
ED3052A	183	183	1	[LUAm1]
ED3052B	200	200	1	[LUAm1]
JU1400	2109	2109	9	[LUAm1, ERTm5, AWRm78, LUAm3, MAM1, LUAm1-HK, ...
JU300	132	132	1	[ERTm5]
JU360	251	251	2	[LUAm1, ERTm2]
JU397	113	113	1	[LUAm1]
JU642	121	121	1	[LUAm1]
MY1	613	613	2	[LUAm1, ERTm5]
MY2	250	250	2	[ERTm5, ERTm2]
MY6	112	112	1	[LUAm1]
N2	2221	2221	10	[LUAm1, ERTm5, ERTm2, AWRm78, LUAm3, MAM1, LUA...
VC20019	323	323	3	[LUAm1, ERTm5, ERTm2]
VC40171	150	150	1	[LUAm1]

	numEmbryos
wormStrain
MY2	21.886179
CB4856	20.750000
AWR145	20.656000
AWR144	19.228000
N2	18.912500
MY6	18.360656
JU360	16.788618
JU300	15.622951
VC20019	15.143678
JU642	14.733333
ED3042	13.147541
ED3052B	12.047297
MY1	11.725581
JU397	11.457143
JU1400	11.139501
AB1	10.903226
ED3052A	10.526667
VC40171	4.586667

	exp.name	strain	spore.strain	spore.species	dose	spores	fixing.date	slide	file	worm.number	area	percent.infected	area.infected	timepoint
0	N2-LUAm1-1.8	N2	LUAm1	N.ferruginous	pulse-72H	1.8	rep1	1	N2.LUAm1.rep1	1	49838.02	18.53	9234.985106	72hpi
1	N2-LUAm1-1.8	N2	LUAm1	N.ferruginous	pulse-72H	1.8	rep1	1	N2.LUAm1.rep1	2	50425.04	0.00	0.000000	72hpi
2	N2-LUAm1-1.8	N2	LUAm1	N.ferruginous	pulse-72H	1.8	rep1	1	N2.LUAm1.rep1	3	45532.67	31.16	14187.979970	72hpi
3	N2-LUAm1-1.8	N2	LUAm1	N.ferruginous	pulse-72H	1.8	rep1	1	N2.LUAm1.rep1	4	46458.55	3.88	1802.591740	72hpi
4	N2-LUAm1-1.8	N2	LUAm1	N.ferruginous	pulse-72H	1.8	rep1	1	N2.LUAm1.rep1	5	49214.73	0.00	0.000000	72hpi

	date	wormStrain	pathogenStrain	doseLevel	numEmbryos
0	190426	AB1	LUAm1	Medium	8.583333
1	190426	AB1	LUAm1	Mock	10.903226
2	190426	CB4856	ERTm5	Medium	17.484375
3	190426	CB4856	ERTm5	Mock	20.750000
4	190426	ED3042	LUAm1	Medium	2.352941
...	...	...	...	...	...
138	200916	N2	ERTm5-96H	Mock	20.140000
139	200918	AWR144	ERTm5	Mock	20.060000
140	200918	AWR145	ERTm5	Mock	21.760000
141	200918	JU1400	ERTm5	Mock	14.140000
142	200918	N2	ERTm5	Mock	21.360000

Introduction to Python for Data Science¶

Lecture 04: Exploratory data analysis and plotting¶

Student Name: Live HTML Test¶

Student ID: NA¶

0.1.0 About Introduction to Python¶

0.1.1 Where is this course headed?¶

0.1.2 How do we get there? Step-by-step.¶

0.1.3 What kind of coding style will we learn?¶

0.2.0 Lecture objectives¶

0.3.0 A legend for text format in Jupyter markdown¶

0.4.0 Lecture and data files used in this course¶

0.4.1 Weekly Lecture and skeleton files¶

0.4.2 Live-coding HTML page¶

0.4.3 Post-lecture PDFs¶

0.4.4 Microsporidia infection data set description¶

0.4.4.1 Dataset 1: /data/embryo_long_merged.csv¶

0.4.4.2 Dataset 2: /data/infection_signal.tsv¶

0.5.0 Packages used in this lesson¶

1.0.0 An introduction to exploratory data analysis¶

1.0.1 Import our data from last week¶

1.1.0 Which worm and pathogen strains are most often used in our experiments?¶

1.1.1 Get a summary of our information¶

1.1.2 Use the .describe() method to summarize non-numeric columns¶

1.1.3 Filter your data with conditional booleans¶

1.1.4 The .describe() method returns a Series object¶

1.1.5 Know the difference between filtering and the .filter() method¶

1.1.6 Use the .query() method to conditional filter your data¶

1.2.0 Which worm strain is most often included in our different experimental groups?¶

1.2.1 Grouping your DataFrame using the .groupby() method¶

1.2.2 Use the .head() method to view rows from each group¶

1.2.3 Subset and apply functions to your grouped DataFrame¶

1.3.0 What is the mean number of embryos produced by uninfected animals in our study?¶

1.3.1 Use sort_values() to organize your data¶

Comprehension Question 1.3.1 Answer:¶

1.4.0 What is the dispersion of the uninfected mean across different sets of replicate experiments?¶

1.4.1 Plan your analysis strategy to avoid mistaken assumptions¶

1.5.0 How many pathogens have been tested on each strain?¶

1.5.1 Use the .apply() method to broadcast a function to individual elements¶

1.5.2 Use .nunique() on a grouped dataframe to return the number of unique elements¶

1.5.3 Use the .agg() method to generate a summary from multiple functions¶

1.5.4 Use different methods for different variables with the .agg() method¶

1.5.5 Use more complex filtering with the .query() method¶

1.5.6 Find the right tool to save yourself time¶

2.0.0 Plotting with the pyplot module¶

2.0.1 Use exploratory plots to assess your data¶

2.0.2 Types of plots¶

2.0.3 Plot components¶

2.1.0 Build a basic barplot with the bar() method¶

2.1.1 Use the figure() function to set your figure size¶

2.1.2 Rotate your x-axis text with the xticks() function¶

2.1.3 Add labels to your plot¶

2.1.4 Alter properties of your barplot¶

3.0.0 Basic visualizations with the seaborn package¶

3.0.1 The seaborn package subdivides plot types into 3 categories¶

3.1.0 Introduction to the Grammar of Graphics¶

3.1.1 The grammar of graphics with seaborn¶

3.1.2 Import infection_signal.tsv¶

3.2.0 Build a scatterplot using the relplot() function¶

3.2.1 Specify colouring by category with the hue parameter¶

3.2.2 Alter your axis scale with the .set() method¶

3.2.3 Facet your data into different plots with the relplot() function¶

3.2.4 Use a continuous variable to colour your data¶

3.2.5 Update your axis using the .set_axis_labels() method¶

3.2.6 Set the shape of your points with the style parameter¶

3.2.7 Using the right colours for your visualization with the color_palette() function¶

3.3.0 Plotting theoretical distributions¶

3.3.1 More on filtering your data with the boolean AND operator: &¶

3.3.2 Look at your distribution as a kernel density estimate with kdeplot() or displot()¶

3.3.3 The category data type reassigns order or meaning to your values¶

3.3.3.1 Reverse the order of a list using slicing notation¶

3.3.3.2 Reorder your categories with reorder_categories()¶

3.3.4 Add a rugplot() to the margin¶

3.4.0 Boxplots provide visual summary statistics of your data¶

3.4.1 Generate a basic boxplot with the catplot() function¶

3.4.2 Rotate your x-axis through the set_xticklabels() method¶

3.4.4 Generate nested boxplots using the hue parameter¶

3.4.5 Facet your crowded data with the row and col parameters¶

3.4.6 Overlay on a faceted plot using map_dataframe()¶

3.5.0 Use the violin plot to visualize distributions¶

3.5.1 Combine a binary grouping with the split parameter¶

1.1.2 Use the `.describe()` method to summarize non-numeric columns¶

1.1.4 The `.describe()` method returns a Series object¶

1.1.5 Know the difference between filtering and the `.filter()` method¶

1.1.6 Use the `.query()` method to conditional filter your data¶

1.2.1 Grouping your DataFrame using the `.groupby()` method¶

1.2.2 Use the `.head()` method to view rows from each group¶

1.3.1 Use `sort_values()` to organize your data¶

1.5.1 Use the `.apply()` method to broadcast a function to individual elements¶

1.5.2 Use `.nunique()` on a grouped dataframe to return the number of unique elements¶

1.5.3 Use the `.agg()` method to generate a summary from multiple functions¶

1.5.4 Use different methods for different variables with the `.agg()` method¶

1.5.5 Use more complex filtering with the `.query()` method¶

2.0.0 Plotting with the `pyplot` module¶

2.1.0 Build a basic barplot with the `bar()` method¶

2.1.1 Use the `figure()` function to set your figure size¶

2.1.2 Rotate your x-axis text with the `xticks()` function¶

3.0.0 Basic visualizations with the `seaborn` package¶

3.0.1 The `seaborn` package subdivides plot types into 3 categories¶

3.1.1 The grammar of graphics with `seaborn`¶

3.1.2 Import `infection_signal.tsv`¶

3.2.0 Build a scatterplot using the `relplot()` function¶

3.2.1 Specify colouring by category with the `hue` parameter¶

3.2.2 Alter your axis scale with the `.set()` method¶

3.2.3 Facet your data into different plots with the `relplot()` function¶

3.2.5 Update your axis using the `.set_axis_labels()` method¶

3.2.6 Set the shape of your points with the `style` parameter¶

3.2.7 Using the right colours for your visualization with the `color_palette()` function¶

3.3.1 More on filtering your data with the boolean AND operator: `&`¶

3.3.2 Look at your distribution as a kernel density estimate with `kdeplot()` or `displot()`¶

3.3.3 The `category` data type reassigns order or meaning to your values¶

3.3.3.2 Reorder your categories with `reorder_categories()`¶

3.3.4 Add a `rugplot()` to the margin¶

3.4.1 Generate a basic boxplot with the `catplot()` function¶

3.4.2 Rotate your x-axis through the `set_xticklabels()` method¶

3.4.4 Generate nested boxplots using the `hue` parameter¶

3.4.5 Facet your crowded data with the `row` and `col` parameters¶

3.4.6 Overlay on a faceted plot using `map_dataframe()`¶

3.5.1 Combine a binary grouping with the `split` parameter¶

4.1.0 Save your figures with `plt.savefig()` method¶

5.1.0 Use `subplot2grid()` to generate a multi-grid plot¶

5.1.1 Use the `Axes` objects as a canvas to plot onto `fig`¶

5.1.2 Add multiple layers to the second `Axes`¶

5.1.5 Adjust spacing between `Axes` using `tight_layout()`¶